from collections import defaultdict, Counter
import gzip, json, os, csv, hashlib, math
from math import sqrt, log, log10, pow

def get_cosine(vec1, vec2):
	intersection = set(vec1.keys()) & set(vec2.keys())
	numerator = sum([vec1[x] * vec2[x] for x in intersection])
	sum1 = sum([vec1[x]**2 for x in vec1.keys()])
	sum2 = sum([vec2[x]**2 for x in vec2.keys()])
	denominator = math.sqrt(sum1) * math.sqrt(sum2)
	if not denominator:
		return 0.0
	else:
		return float(numerator) / denominator


def merge(vec1, vec2):
	result = Counter()
	for term in set(vec1.keys() + vec2.keys()):
		result[term] = float(vec1[term] + vec2[term]) / 2
	return result

print "reading data from local copy"


print "clustering books"

BOOKS = sorted([f.split(".")[0] for f in os.listdir("BOOK_TF/")])

PROD_CLUSTERS = defaultdict(list)
PROD_CENTROIDS = defaultdict(Counter)

book = BOOKS.pop()
VECTOR = Counter(json.loads(gzip.open("BOOK_TF/"+book+".json.gz",'r').readline()))
PROD_CLUSTERS[hashlib.md5(book).hexdigest()].append(book)
PROD_CENTROIDS[hashlib.md5(book).hexdigest()] = VECTOR

#for book in BOOKS[1:]:
while len(BOOKS) > 0:
	book = BOOKS.pop()
	VECTOR = Counter(json.loads(gzip.open("BOOK_TF/"+book+".json.gz",'r').readline()))
	if len(VECTOR) == 0:
		os.remove("BOOK_TF/"+book+".json.gz")
	else:
		temp = Counter()
		for centroid in PROD_CENTROIDS:	# check book against all clusters
			temp[centroid]  = get_cosine(PROD_CENTROIDS[centroid],VECTOR)
		(c,m) = temp.most_common(1)[0]	# find the cluster with the maximum similarity
		if m > 0.9 :			# if similarity is more than 0.9 add user to the cluster
			PROD_CLUSTERS[c].append(book)
			PROD_CENTROIDS[c] = merge(PROD_CENTROIDS[c],VECTOR)
		else:				# else create new cluster
			PROD_CLUSTERS[hashlib.md5(book).hexdigest()].append(book)
			PROD_CENTROIDS[hashlib.md5(book).hexdigest()] = VECTOR
	print str(len(BOOKS)) + " books left to cluster"


clustfile = gzip.open("PROD_CLUSTERS.json.gz",'w')
centsfile = gzip.open("PROD_CENTROIDS.json.gz",'w')
for cluster in PROD_CLUSTERS.keys():
	clustfile.write(json.dumps([cluster,PROD_CLUSTERS[cluster]])+"\n")
	centsfile.write(json.dumps([cluster,PROD_CENTROIDS[cluster]])+"\n")
clustfile.close()
centsfile.close()


